

<!DOCTYPE html>
<html lang="zh-CN" data-default-color-scheme=auto>



<head>
  <meta charset="UTF-8">
  <link rel="apple-touch-icon" sizes="76x76" href="/img/fluid.png">
  <link rel="icon" href="/img/fluid.png">
  <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=5.0, shrink-to-fit=no">
  <meta http-equiv="x-ua-compatible" content="ie=edge">
  
  <meta name="theme-color" content="#2f4154">
  <meta name="author" content="John Doe">
  <meta name="keywords" content="">
  
    <meta name="description" content="参考资料 视频资料 一、HDFS 背景以及定义 背景： 数据量越来越大，操作系统存不了所有数据，需要能管理多台机器上的文件的系统，即分布式文件管理系统，HDFS就是其中的一种。 定义： HDFS，全称 Hadoop Distributed File System，是一个文件系统，通过 目录树 来定位文件，具有分布性，是由多个服务器联合起来实现的存储功能。 适用场景：一次写入，多次读出。 即一个文件">
<meta property="og:type" content="article">
<meta property="og:title" content="Hadoop _ HDFS学习笔记（一）HDFS优缺点、NN+DN+2NN组成架构、block文件块 _ Shell操作 _ HDFS常用命令汇总">
<meta property="og:url" content="http://example.com/2022/01/17/Hadoop%20_%20HDFS%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0%EF%BC%88%E4%B8%80%EF%BC%89HDFS%E4%BC%98%E7%BC%BA%E7%82%B9%E3%80%81NN+DN+2NN%E7%BB%84%E6%88%90%E6%9E%B6%E6%9E%84%E3%80%81block%E6%96%87%E4%BB%B6%E5%9D%97%20_%20Shell%E6%93%8D%E4%BD%9C%20_%20HDFS%E5%B8%B8%E7%94%A8%E5%91%BD%E4%BB%A4%E6%B1%87%E6%80%BB/index.html">
<meta property="og:site_name" content="Hexo">
<meta property="og:description" content="参考资料 视频资料 一、HDFS 背景以及定义 背景： 数据量越来越大，操作系统存不了所有数据，需要能管理多台机器上的文件的系统，即分布式文件管理系统，HDFS就是其中的一种。 定义： HDFS，全称 Hadoop Distributed File System，是一个文件系统，通过 目录树 来定位文件，具有分布性，是由多个服务器联合起来实现的存储功能。 适用场景：一次写入，多次读出。 即一个文件">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://img-blog.csdnimg.cn/2de82b9db9114560b904e8590c8b775e.png">
<meta property="og:image" content="https://img-blog.csdnimg.cn/5f94c4b3a9334e1386ecde9b8ae08961.png">
<meta property="article:published_time" content="2022-01-17T05:54:28.000Z">
<meta property="article:modified_time" content="2022-08-22T15:44:00.201Z">
<meta property="article:author" content="John Doe">
<meta property="article:tag" content="HDFS">
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:image" content="https://img-blog.csdnimg.cn/2de82b9db9114560b904e8590c8b775e.png">
  
  
  
  <title>Hadoop _ HDFS学习笔记（一）HDFS优缺点、NN+DN+2NN组成架构、block文件块 _ Shell操作 _ HDFS常用命令汇总 - Hexo</title>

  <link  rel="stylesheet" href="https://lib.baomitu.com/twitter-bootstrap/4.6.1/css/bootstrap.min.css" />



  <link  rel="stylesheet" href="https://lib.baomitu.com/github-markdown-css/4.0.0/github-markdown.min.css" />

  <link  rel="stylesheet" href="https://lib.baomitu.com/hint.css/2.7.0/hint.min.css" />

  <link  rel="stylesheet" href="https://lib.baomitu.com/fancybox/3.5.7/jquery.fancybox.min.css" />



<!-- 主题依赖的图标库，不要自行修改 -->
<!-- Do not modify the link that theme dependent icons -->

<link rel="stylesheet" href="//at.alicdn.com/t/font_1749284_hj8rtnfg7um.css">



<link rel="stylesheet" href="//at.alicdn.com/t/font_1736178_lbnruvf0jn.css">


<link  rel="stylesheet" href="/css/main.css" />


  <link id="highlight-css" rel="stylesheet" href="/css/highlight.css" />
  
    <link id="highlight-css-dark" rel="stylesheet" href="/css/highlight-dark.css" />
  




  <script id="fluid-configs">
    var Fluid = window.Fluid || {};
    Fluid.ctx = Object.assign({}, Fluid.ctx)
    var CONFIG = {"hostname":"example.com","root":"/","version":"1.9.2","typing":{"enable":true,"typeSpeed":70,"cursorChar":"_","loop":false,"scope":[]},"anchorjs":{"enable":true,"element":"h1,h2,h3,h4,h5,h6","placement":"left","visible":"hover","icon":""},"progressbar":{"enable":true,"height_px":3,"color":"#29d","options":{"showSpinner":false,"trickleSpeed":100}},"code_language":{"enable":true,"default":"TEXT"},"copy_btn":true,"image_caption":{"enable":true},"image_zoom":{"enable":true,"img_url_replace":["",""]},"toc":{"enable":true,"placement":"right","headingSelector":"h1,h2,h3,h4,h5,h6","collapseDepth":0},"lazyload":{"enable":true,"loading_img":"/img/loading.gif","onlypost":false,"offset_factor":2},"web_analytics":{"enable":false,"follow_dnt":true,"baidu":null,"google":null,"gtag":null,"tencent":{"sid":null,"cid":null},"woyaola":null,"cnzz":null,"leancloud":{"app_id":null,"app_key":null,"server_url":null,"path":"window.location.pathname","ignore_local":false}},"search_path":"/local-search.xml"};

    if (CONFIG.web_analytics.follow_dnt) {
      var dntVal = navigator.doNotTrack || window.doNotTrack || navigator.msDoNotTrack;
      Fluid.ctx.dnt = dntVal && (dntVal.startsWith('1') || dntVal.startsWith('yes') || dntVal.startsWith('on'));
    }
  </script>
  <script  src="/js/utils.js" ></script>
  <script  src="/js/color-schema.js" ></script>
  


  
<meta name="generator" content="Hexo 6.2.0"></head>


<body>
  

  <header>
    

<div class="header-inner" style="height: 70vh;">
  <nav id="navbar" class="navbar fixed-top  navbar-expand-lg navbar-dark scrolling-navbar">
  <div class="container">
    <a class="navbar-brand" href="/">
      <strong>尤 Ni&#39;s Blog</strong>
    </a>

    <button id="navbar-toggler-btn" class="navbar-toggler" type="button" data-toggle="collapse"
            data-target="#navbarSupportedContent"
            aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
      <div class="animated-icon"><span></span><span></span><span></span></div>
    </button>

    <!-- Collapsible content -->
    <div class="collapse navbar-collapse" id="navbarSupportedContent">
      <ul class="navbar-nav ml-auto text-center">
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/">
                <i class="iconfont icon-home-fill"></i>
                首页
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/archives/">
                <i class="iconfont icon-archive-fill"></i>
                归档
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/categories/">
                <i class="iconfont icon-category-fill"></i>
                分类
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/tags/">
                <i class="iconfont icon-tags-fill"></i>
                标签
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/about/">
                <i class="iconfont icon-user-fill"></i>
                关于
              </a>
            </li>
          
        
        
          <li class="nav-item" id="search-btn">
            <a class="nav-link" target="_self" href="javascript:;" data-toggle="modal" data-target="#modalSearch" aria-label="Search">
              &nbsp;<i class="iconfont icon-search"></i>&nbsp;
            </a>
          </li>
          
        
        
          <li class="nav-item" id="color-toggle-btn">
            <a class="nav-link" target="_self" href="javascript:;" aria-label="Color Toggle">&nbsp;<i
                class="iconfont icon-dark" id="color-toggle-icon"></i>&nbsp;</a>
          </li>
        
      </ul>
    </div>
  </div>
</nav>

  

<div id="banner" class="banner" parallax=true
     style="background: url('/img/default.png') no-repeat center center; background-size: cover;">
  <div class="full-bg-img">
    <div class="mask flex-center" style="background-color: rgba(0, 0, 0, 0.3)">
      <div class="banner-text text-center fade-in-up">
        <div class="h2">
          
            <span id="subtitle" data-typed-text="Hadoop _ HDFS学习笔记（一）HDFS优缺点、NN+DN+2NN组成架构、block文件块 _ Shell操作 _ HDFS常用命令汇总"></span>
          
        </div>

        
          
  <div class="mt-3">
    
      <span class="post-meta mr-2">
        <i class="iconfont icon-author" aria-hidden="true"></i>
        John Doe
      </span>
    
    
      <span class="post-meta">
        <i class="iconfont icon-date-fill" aria-hidden="true"></i>
        <time datetime="2022-01-17 13:54" pubdate>
          2022年1月17日 下午
        </time>
      </span>
    
  </div>

  <div class="mt-1">
    
      <span class="post-meta mr-2">
        <i class="iconfont icon-chart"></i>
        
          5.9k 字
        
      </span>
    

    
      <span class="post-meta mr-2">
        <i class="iconfont icon-clock-fill"></i>
        
        
        
          49 分钟
        
      </span>
    

    
    
  </div>


        
      </div>

      
    </div>
  </div>
</div>

</div>

  </header>

  <main>
    
      

<div class="container-fluid nopadding-x">
  <div class="row nomargin-x">
    <div class="side-col d-none d-lg-block col-lg-2">
      

    </div>

    <div class="col-lg-8 nopadding-x-md">
      <div class="container nopadding-x-md" id="board-ctn">
        <div id="board">
          <article class="post-content mx-auto">
            <!-- SEO header -->
            <h1 style="display: none">Hadoop _ HDFS学习笔记（一）HDFS优缺点、NN+DN+2NN组成架构、block文件块 _ Shell操作 _ HDFS常用命令汇总</h1>
            
              <p class="note note-info">
                
                  
                    本文最后更新于：1 小时前
                  
                
              </p>
            
            
              <div class="markdown-body">
                
                <h1 id="参考资料"><a href="#参考资料" class="headerlink" title="参考资料"></a>参考资料</h1><hr>
<p><a target="_blank" rel="noopener" href="https://www.bilibili.com/video/BV1Qp4y1n7EN?p=41">视频资料</a></p>
<h1 id="一、HDFS-背景以及定义"><a href="#一、HDFS-背景以及定义" class="headerlink" title="一、HDFS 背景以及定义"></a>一、HDFS 背景以及定义</h1><hr>
<p><strong>背景：</strong> 数据量越来越大，操作系统存不了所有数据，需要能管理多台机器上的文件的系统，即分布式文件管理系统，HDFS就是其中的一种。</p>
<p><strong>定义：</strong> HDFS，全称 Hadoop Distributed File System，是一个文件系统，通过 <strong>目录树</strong> 来定位文件，具有分布性，是由多个服务器联合起来实现的存储功能。</p>
<p><strong>适用场景：一次写入，多次读出。</strong> 即一个文件经过创建、写入和关闭后就不需要改变的情况。</p>
<h1 id="二、HDFS-优缺点"><a href="#二、HDFS-优缺点" class="headerlink" title="二、HDFS 优缺点"></a>二、HDFS 优缺点</h1><hr>
<h2 id="2-1-优点"><a href="#2-1-优点" class="headerlink" title="2.1 优点"></a>2.1 优点</h2><ul>
<li><p><strong>高容错性：</strong> 数据自动保存多个副本，可增加副本来提高容错性，某个副本丢失后，可自动恢复。</p>
</li>
<li><p><strong>适合处理大数据：</strong>（1）数据规模：能处理达到GB、TB甚至PB级别的数据；（2）能处理百万规模以上的文件数量。</p>
</li>
<li><p><strong>可构建在廉价机器上：</strong> 通过多副本机制，提高可靠性。</p>
</li>
</ul>
<h2 id="2-2-缺点"><a href="#2-2-缺点" class="headerlink" title="2.2 缺点"></a>2.2 缺点</h2><ul>
<li><strong>不适合低延时</strong>数据访问，比如无法做到毫秒级存储数据。</li>
<li><strong>无法高效</strong>的对<strong>大量小文件</strong>进行存储。<ul>
<li>存储大量小文件时，会占用 NameNode节点大量的内存来存储文件目录和块信息。</li>
<li>小文件存储的寻址时间会超过读读取时间，违反了HDFS设计理念。</li>
</ul>
</li>
<li><strong>不支持</strong>并发写入、文件随机<strong>修改</strong> <ul>
<li>不允许多个线程对同一个文件进行写操作</li>
<li>仅支持数据 <strong>append（追加）</strong>，不允许文件的随机修改</li>
</ul>
</li>
</ul>
<h1 id="三、HDFS组成架构"><a href="#三、HDFS组成架构" class="headerlink" title="三、HDFS组成架构"></a>三、HDFS组成架构</h1><hr>
<p>图片来自 <a target="_blank" rel="noopener" href="https://hadoop.apache.org/docs/r3.1.3/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html">Hadoop3.1.3 HDFS官方文档</a><br><img src="https://img-blog.csdnimg.cn/2de82b9db9114560b904e8590c8b775e.png" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"></p>
<h2 id="3-1-NameNode（NN）"><a href="#3-1-NameNode（NN）" class="headerlink" title="3.1 NameNode（NN）"></a>3.1 NameNode（NN）</h2><p>Master，是主管、管理者的身份</p>
<ol>
<li>管理HDFS的名称空间</li>
<li>配置副本策略</li>
<li>管理数据块（Block）映射信息</li>
<li>处理客户端读写请求</li>
</ol>
<h2 id="3-2-DataNode-（DN）"><a href="#3-2-DataNode-（DN）" class="headerlink" title="3.2 DataNode （DN）"></a>3.2 DataNode （DN）</h2><p>Slave，执行NameNode下达的命令</p>
<ol>
<li>存储实际的数据块</li>
<li>执行数据块的读 &#x2F; 写操作</li>
</ol>
<h2 id="3-3-Client"><a href="#3-3-Client" class="headerlink" title="3.3 Client"></a>3.3 Client</h2><p>客户端，shell客户端或者web客户端</p>
<ol>
<li>文件切分。文件上传到 HDFS 时， Client 将文件切分成一个一个的Block，然后上传</li>
<li>与 NameNode 交互，获取文件的位置信息</li>
<li>与 DataNode 交互，读取或者写入数据</li>
<li>Client 提供一些命令来管理 HDFS，比如 NameNode 格式化</li>
<li>Client 可以通过一些命令来访问 HDFS，比如对 HDFS 增删改查操作</li>
</ol>
<h2 id="3-4-SecondaryNameNode"><a href="#3-4-SecondaryNameNode" class="headerlink" title="3.4 SecondaryNameNode"></a>3.4 SecondaryNameNode</h2><p><strong>不是</strong> NameNode的热备，当NameNode挂掉时，并不能马上替换NameNode并提供服务。</p>
<ol>
<li>辅助 NameNode，分担其工作量，比如定期合并 Fsimage 和 Edits，并推送给NameNode</li>
<li>在紧急情况下，可辅助恢复NameNode的状态</li>
</ol>
<h1 id="四、HDFS-文件块大小"><a href="#四、HDFS-文件块大小" class="headerlink" title="四、HDFS 文件块大小"></a>四、HDFS 文件块大小</h1><hr>
<h2 id="4-1-默认块大小"><a href="#4-1-默认块大小" class="headerlink" title="4.1 默认块大小"></a>4.1 默认块大小</h2><p>HDFS中的文件在物理上是分块存储（Block），块的大小可通过配置参数（<code>dfs.blocksize</code>）来决定，在<code>hdfs-default.xml</code><a target="_blank" rel="noopener" href="https://gitee.com/ccuni/hadoop-learn/blob/master/hadoop3.1.3-default-config/hdfs-default.xml">[参考]</a>中的默认配置如下：</p>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><code class="hljs xml"><span class="hljs-tag">&lt;<span class="hljs-name">property</span>&gt;</span><br>  <span class="hljs-tag">&lt;<span class="hljs-name">name</span>&gt;</span>dfs.blocksize<span class="hljs-tag">&lt;/<span class="hljs-name">name</span>&gt;</span><br>  <span class="hljs-tag">&lt;<span class="hljs-name">value</span>&gt;</span>134217728<span class="hljs-tag">&lt;/<span class="hljs-name">value</span>&gt;</span><br>  <span class="hljs-tag">&lt;<span class="hljs-name">description</span>&gt;</span><br>      The default block size for new files, in bytes.<br>      You can use the following suffix (case insensitive):<br>      k(kilo), m(mega), g(giga), t(tera), p(peta), e(exa) to specify the size (such as 128k, 512m, 1g, etc.),<br>      Or provide complete size in bytes (such as 134217728 for 128 MB).<br>  <span class="hljs-tag">&lt;/<span class="hljs-name">description</span>&gt;</span><br><span class="hljs-tag">&lt;/<span class="hljs-name">property</span>&gt;</span><br></code></pre></td></tr></table></figure>

<p>块大小默认为134217728 单位是 Byte 字节</p>
<p>换算成 MB 就是： 134217728 &#x2F; 1024 &#x2F; 1024  &#x3D; 128 MB</p>
<p>在 Hadoop1.x版本的时候块大小是 64MB，之后的2.x和3.x都是128MB</p>
<p>若寻址时间约为10ms，即查找到目标block的时间为10ms。</p>
<p>寻址时间为传输时间的1%时，则为最佳状态，因此传输时间为：10ms &#x2F; 0.01 &#x3D; 1000ms&#x3D;1s</p>
<p>最佳状态：0.01 &#x3D; 寻址时间 &#x2F; 传输时间</p>
<p>又因为目前磁盘的传输速率普遍为 100MB&#x2F;s</p>
<p>由 1s * 100 MB&#x2F;s 计算结果为 100 MB，所以块大小默认为128MB是比较合适的。</p>
<h2 id="4-2-文件块大小问题"><a href="#4-2-文件块大小问题" class="headerlink" title="4.2 文件块大小问题"></a>4.2 文件块大小问题</h2><p><strong>HDFS块设置太小导致的问题:</strong></p>
<ul>
<li>增加寻址时间，程序会花更多时间去寻找块的开始位置</li>
</ul>
<p><strong>HDFS块设置太大导致的问题：</strong></p>
<ul>
<li>从<strong>磁盘传输数据的时间</strong>会明显大于<strong>定位这块开始位置所需的时间</strong>  ，导致程序在处理这块数据时会非常慢。</li>
</ul>
<h1 id="五、HDFS-的Shell操作"><a href="#五、HDFS-的Shell操作" class="headerlink" title="五、HDFS 的Shell操作"></a>五、HDFS 的Shell操作</h1><hr>
<p><strong>基本语法（有两种）：</strong></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><code class="hljs shell">hadoop fs 具体命令<br><br>hdfs dfs 具体命令<br></code></pre></td></tr></table></figure>

<figure class="highlight inform7"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br></pre></td><td class="code"><pre><code class="hljs inform7"><span class="hljs-comment">[root@hadoop101 ~]</span># hadoop fs<br>Usage: hadoop fs <span class="hljs-comment">[generic options]</span><br>	<span class="hljs-comment">[-appendToFile &lt;localsrc&gt; ... &lt;dst&gt;]</span><br>	<span class="hljs-comment">[-cat <span class="hljs-comment">[-ignoreCrc]</span> &lt;src&gt; ...]</span><br>	<span class="hljs-comment">[-checksum &lt;src&gt; ...]</span><br>	<span class="hljs-comment">[-chgrp <span class="hljs-comment">[-R]</span> GROUP PATH...]</span><br>	<span class="hljs-comment">[-chmod <span class="hljs-comment">[-R]</span> &lt;MODE<span class="hljs-comment">[,MODE]</span>... | OCTALMODE&gt; PATH...]</span><br>	<span class="hljs-comment">[-chown <span class="hljs-comment">[-R]</span> <span class="hljs-comment">[OWNER]</span><span class="hljs-comment">[:<span class="hljs-comment">[GROUP]</span>]</span> PATH...]</span><br>	<span class="hljs-comment">[-copyFromLocal <span class="hljs-comment">[-f]</span> <span class="hljs-comment">[-p]</span> <span class="hljs-comment">[-l]</span> <span class="hljs-comment">[-d]</span> <span class="hljs-comment">[-t &lt;thread count&gt;]</span> &lt;localsrc&gt; ... &lt;dst&gt;]</span><br>	<span class="hljs-comment">[-copyToLocal <span class="hljs-comment">[-f]</span> <span class="hljs-comment">[-p]</span> <span class="hljs-comment">[-ignoreCrc]</span> <span class="hljs-comment">[-crc]</span> &lt;src&gt; ... &lt;localdst&gt;]</span><br>	<span class="hljs-comment">[-count <span class="hljs-comment">[-q]</span> <span class="hljs-comment">[-h]</span> <span class="hljs-comment">[-v]</span> <span class="hljs-comment">[-t <span class="hljs-comment">[&lt;storage type&gt;]</span>]</span> <span class="hljs-comment">[-u]</span> <span class="hljs-comment">[-x]</span> <span class="hljs-comment">[-e]</span> &lt;path&gt; ...]</span><br>	<span class="hljs-comment">[-cp <span class="hljs-comment">[-f]</span> <span class="hljs-comment">[-p | -p<span class="hljs-comment">[topax]</span>]</span> <span class="hljs-comment">[-d]</span> &lt;src&gt; ... &lt;dst&gt;]</span><br>	<span class="hljs-comment">[-createSnapshot &lt;snapshotDir&gt; <span class="hljs-comment">[&lt;snapshotName&gt;]</span>]</span><br>	<span class="hljs-comment">[-deleteSnapshot &lt;snapshotDir&gt; &lt;snapshotName&gt;]</span><br>	<span class="hljs-comment">[-df <span class="hljs-comment">[-h]</span> <span class="hljs-comment">[&lt;path&gt; ...]</span>]</span><br>	<span class="hljs-comment">[-du <span class="hljs-comment">[-s]</span> <span class="hljs-comment">[-h]</span> <span class="hljs-comment">[-v]</span> <span class="hljs-comment">[-x]</span> &lt;path&gt; ...]</span><br>	<span class="hljs-comment">[-expunge]</span><br>	<span class="hljs-comment">[-find &lt;path&gt; ... &lt;expression&gt; ...]</span><br>	<span class="hljs-comment">[-get <span class="hljs-comment">[-f]</span> <span class="hljs-comment">[-p]</span> <span class="hljs-comment">[-ignoreCrc]</span> <span class="hljs-comment">[-crc]</span> &lt;src&gt; ... &lt;localdst&gt;]</span><br>	<span class="hljs-comment">[-getfacl <span class="hljs-comment">[-R]</span> &lt;path&gt;]</span><br>	<span class="hljs-comment">[-getfattr <span class="hljs-comment">[-R]</span> &#123;-n name | -d&#125; <span class="hljs-comment">[-e en]</span> &lt;path&gt;]</span><br>	<span class="hljs-comment">[-getmerge <span class="hljs-comment">[-nl]</span> <span class="hljs-comment">[-skip-empty-file]</span> &lt;src&gt; &lt;localdst&gt;]</span><br>	<span class="hljs-comment">[-head &lt;file&gt;]</span><br>	<span class="hljs-comment">[-help <span class="hljs-comment">[cmd ...]</span>]</span><br>	<span class="hljs-comment">[-ls <span class="hljs-comment">[-C]</span> <span class="hljs-comment">[-d]</span> <span class="hljs-comment">[-h]</span> <span class="hljs-comment">[-q]</span> <span class="hljs-comment">[-R]</span> <span class="hljs-comment">[-t]</span> <span class="hljs-comment">[-S]</span> <span class="hljs-comment">[-r]</span> <span class="hljs-comment">[-u]</span> <span class="hljs-comment">[-e]</span> <span class="hljs-comment">[&lt;path&gt; ...]</span>]</span><br>	<span class="hljs-comment">[-mkdir <span class="hljs-comment">[-p]</span> &lt;path&gt; ...]</span><br>	<span class="hljs-comment">[-moveFromLocal &lt;localsrc&gt; ... &lt;dst&gt;]</span><br>	<span class="hljs-comment">[-moveToLocal &lt;src&gt; &lt;localdst&gt;]</span><br>	<span class="hljs-comment">[-mv &lt;src&gt; ... &lt;dst&gt;]</span><br>	<span class="hljs-comment">[-put <span class="hljs-comment">[-f]</span> <span class="hljs-comment">[-p]</span> <span class="hljs-comment">[-l]</span> <span class="hljs-comment">[-d]</span> &lt;localsrc&gt; ... &lt;dst&gt;]</span><br>	<span class="hljs-comment">[-renameSnapshot &lt;snapshotDir&gt; &lt;oldName&gt; &lt;newName&gt;]</span><br>	<span class="hljs-comment">[-rm <span class="hljs-comment">[-f]</span> <span class="hljs-comment">[-r|-R]</span> <span class="hljs-comment">[-skipTrash]</span> <span class="hljs-comment">[-safely]</span> &lt;src&gt; ...]</span><br>	<span class="hljs-comment">[-rmdir <span class="hljs-comment">[--ignore-fail-on-non-empty]</span> &lt;dir&gt; ...]</span><br>	<span class="hljs-comment">[-setfacl <span class="hljs-comment">[-R]</span> <span class="hljs-comment">[&#123;-b|-k&#125; &#123;-m|-x &lt;acl_spec&gt;&#125; &lt;path&gt;]</span>|<span class="hljs-comment">[--set &lt;acl_spec&gt; &lt;path&gt;]</span>]</span><br>	<span class="hljs-comment">[-setfattr &#123;-n name <span class="hljs-comment">[-v value]</span> | -x name&#125; &lt;path&gt;]</span><br>	<span class="hljs-comment">[-setrep <span class="hljs-comment">[-R]</span> <span class="hljs-comment">[-w]</span> &lt;rep&gt; &lt;path&gt; ...]</span><br>	<span class="hljs-comment">[-stat <span class="hljs-comment">[format]</span> &lt;path&gt; ...]</span><br>	<span class="hljs-comment">[-tail <span class="hljs-comment">[-f]</span> <span class="hljs-comment">[-s &lt;sleep interval&gt;]</span> &lt;file&gt;]</span><br>	<span class="hljs-comment">[-test -<span class="hljs-comment">[defsz]</span> &lt;path&gt;]</span><br>	<span class="hljs-comment">[-text <span class="hljs-comment">[-ignoreCrc]</span> &lt;src&gt; ...]</span><br>	<span class="hljs-comment">[-touch <span class="hljs-comment">[-a]</span> <span class="hljs-comment">[-m]</span> <span class="hljs-comment">[-t TIMESTAMP ]</span> <span class="hljs-comment">[-c]</span> &lt;path&gt; ...]</span><br>	<span class="hljs-comment">[-touchz &lt;path&gt; ...]</span><br>	<span class="hljs-comment">[-truncate <span class="hljs-comment">[-w]</span> &lt;length&gt; &lt;path&gt; ...]</span><br>	<span class="hljs-comment">[-usage <span class="hljs-comment">[cmd ...]</span>]</span><br><br>Generic options supported <span class="hljs-keyword">are</span>:<br>-conf &lt;configuration file&gt;        specify an application configuration file<br>-D &lt;property=value&gt;               define a value for a given property<br>-fs &lt;file:///|hdfs://namenode:port&gt; specify default filesystem URL to use, overrides &#x27;fs.defaultFS&#x27; property from configurations.<br>-jt &lt;local|resourcemanager:port&gt;  specify a ResourceManager<br>-files &lt;file1,...&gt;                specify a comma-separated list <span class="hljs-keyword">of</span> files to be copied to the map reduce cluster<br>-libjars &lt;jar1,...&gt;               specify a comma-separated list <span class="hljs-keyword">of</span> jar files to be included in the classpath<br>-archives &lt;archive1,...&gt;          specify a comma-separated list <span class="hljs-keyword">of</span> archives to be unarchived on the compute machines<br><br>The general command line syntax <span class="hljs-keyword">is</span>:<br>command <span class="hljs-comment">[genericOptions]</span> <span class="hljs-comment">[commandOptions]</span><br><br></code></pre></td></tr></table></figure>

<h2 id="5-1-准备工作"><a href="#5-1-准备工作" class="headerlink" title="5.1 准备工作"></a>5.1 准备工作</h2><p>在NameNode节点启动 HDFS就行，因为暂时没用到MapReduce执行任务，所以Yarn可以不启动。</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><code class="hljs shell">cd $HADOOP_HOME<br>./sbin/start-dfs.sh<br></code></pre></td></tr></table></figure>

<h2 id="5-2-HDFS命令大全"><a href="#5-2-HDFS命令大全" class="headerlink" title="5.2 HDFS命令大全"></a>5.2 HDFS命令大全</h2><p>表中命令前需加<code>hadoop fs</code> 或者 <code>hdfs dfs</code>，注意fs&#x2F;dfs后有一个空格。</p>
<h3 id="5-2-1-上传"><a href="#5-2-1-上传" class="headerlink" title="5.2.1 上传"></a>5.2.1 上传</h3><table>
<thead>
<tr>
<th align="left">命令</th>
<th align="left">作用描述</th>
</tr>
</thead>
<tbody><tr>
<td align="left"><code>-help [命令]</code></td>
<td align="left">查看某个命令的使用提示</td>
</tr>
<tr>
<td align="left"><code>-overFromLocal [本地文件位置] [HDFS文件目录]</code></td>
<td align="left">将本地文件<strong>移动</strong>到HDFS</td>
</tr>
<tr>
<td align="left"><code> -copyFromLocal  [本地文件位置] [HDFS文件目录]</code></td>
<td align="left">将本地文件<strong>复制</strong>到HDFS</td>
</tr>
<tr>
<td align="left"><code>-put [本地文件位置] [HDFS文件目录]</code></td>
<td align="left">将本地文件<strong>复制</strong>到HDFS，等同于<code>-copyFromLocal</code>命令</td>
</tr>
<tr>
<td align="left"><code>-appendToFile [本地文件位置] [HDFS文件位置] </code></td>
<td align="left">追加一个文件到已存在的文件末尾</td>
</tr>
</tbody></table>
<h3 id="5-2-2-下载"><a href="#5-2-2-下载" class="headerlink" title="5.2.2 下载"></a>5.2.2 下载</h3><table>
<thead>
<tr>
<th align="left">命令</th>
<th align="left">作用描述</th>
</tr>
</thead>
<tbody><tr>
<td align="left"><code>-copyToLocal [HDFS文件位置] [本地文件目录]</code></td>
<td align="left">将HDFS的文件拷贝到本地</td>
</tr>
<tr>
<td align="left"><code>-get [HDFS文件位置] [本地文件目录]</code></td>
<td align="left">将HDFS的文件拷贝到本地，等同于<code>-copyToLocal </code>命令</td>
</tr>
</tbody></table>
<h3 id="5-2-3-其他常用命令"><a href="#5-2-3-其他常用命令" class="headerlink" title="5.2.3 其他常用命令"></a>5.2.3 其他常用命令</h3><table>
<thead>
<tr>
<th align="left">命令</th>
<th align="left">作用描述</th>
</tr>
</thead>
<tbody><tr>
<td align="left"><code>-ls [HDFS文件目录]</code></td>
<td align="left">查看HDFS目录的信息</td>
</tr>
<tr>
<td align="left"><code>-cat [HDFS文件位置]</code></td>
<td align="left">显示HDFS文件内容</td>
</tr>
<tr>
<td align="left"><code>-chgrp/-chmod/-chown [HDFS文件位置]</code></td>
<td align="left">和LInux文件系统一样，修改HDFS指定文件的权限</td>
</tr>
<tr>
<td align="left"><code>-mkdir [HDFS文件位置]</code></td>
<td align="left">在HDFS指定位置创建文件夹</td>
</tr>
<tr>
<td align="left"><code>-cp [HDFS文件1的位置] [HDFS文件目录]</code></td>
<td align="left">将HDFS的一个文件<strong>拷贝</strong>到另一个文件路径</td>
</tr>
<tr>
<td align="left"><code>-mv [HDFS文件1的位置] [HDFS文件目录]</code></td>
<td align="left">将HDFS的一个文件<strong>移动</strong>到另一个文件路径</td>
</tr>
<tr>
<td align="left"><code>-tail [HDFS文件位置]</code></td>
<td align="left">显示HDFS中指定文件的末尾1KB大小的数据</td>
</tr>
<tr>
<td align="left"><code>-rm [HDFS文件位置] </code></td>
<td align="left">HDFS删除文件或文件夹</td>
</tr>
<tr>
<td align="left"><code>-rm -r [HDFS文件位置</code></td>
<td align="left">HDFS递归删除目录及目录里面的内容</td>
</tr>
<tr>
<td align="left"><code>-du [HDFS文件目录]</code></td>
<td align="left">HDFS 统计文件夹的大小信息，单位Byte字节</td>
</tr>
</tbody></table>
<p><strong>注：</strong> du命令返回的结果通常是在当前节点的大小和所有副本里的总大小，单位是Byte字节，比如：<br>![在这里插入图片描述](<a target="_blank" rel="noopener" href="https://img-blog.csdnimg.cn/aed744b7e7884e42afaa3c12dc4e28af.png#">https://img-blog.csdnimg.cn/aed744b7e7884e42afaa3c12dc4e28af.png#</a> &#x3D;80%x)<br>其中108表示在当前节点的大小为108Byte，324则表示在所有节点里的大小为324Byte</p>
<p>同时能说明副本个数为3，刚好 108 * 3 &#x3D; 324.</p>
<h3 id="特殊命令"><a href="#特殊命令" class="headerlink" title="特殊命令"></a>特殊命令</h3><p><code>hadoop dfs -setrep [副本个数] [HDFS文件位置]</code>  设置 HDFS中文件的副本数量</p>
<p>设置的副本数会记录在 NameNode的元数据中，是否真的能达到指定的副本个数还需看 DataNode节点的数量。比如集群目前有3个节点，所以实际上最多就3个副本。<br><img src="https://img-blog.csdnimg.cn/5f94c4b3a9334e1386ecde9b8ae08961.png" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"></p>

                
              </div>
            
            <hr/>
            <div>
              <div class="post-metas my-3">
  
    <div class="post-meta mr-3 d-flex align-items-center">
      <i class="iconfont icon-category"></i>
      

<span class="category-chains">
  
  
    
      <span class="category-chain">
        
  <a href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/" class="category-chain-item">大数据</a>
  
  

      </span>
    
  
</span>

    </div>
  
  
    <div class="post-meta">
      <i class="iconfont icon-tags"></i>
      
        <a href="/tags/HDFS/">#HDFS</a>
      
    </div>
  
</div>


              
  

  <div class="license-box my-3">
    <div class="license-title">
      <div>Hadoop _ HDFS学习笔记（一）HDFS优缺点、NN+DN+2NN组成架构、block文件块 _ Shell操作 _ HDFS常用命令汇总</div>
      <div>http://example.com/2022/01/17/Hadoop _ HDFS学习笔记（一）HDFS优缺点、NN+DN+2NN组成架构、block文件块 _ Shell操作 _ HDFS常用命令汇总/</div>
    </div>
    <div class="license-meta">
      
        <div class="license-meta-item">
          <div>作者</div>
          <div>John Doe</div>
        </div>
      
      
        <div class="license-meta-item license-meta-date">
          <div>发布于</div>
          <div>2022年1月17日</div>
        </div>
      
      
      <div class="license-meta-item">
        <div>许可协议</div>
        <div>
          
            
            
              <a target="_blank" href="https://creativecommons.org/licenses/by/4.0/">
              <span class="hint--top hint--rounded" aria-label="BY - 署名">
                <i class="iconfont icon-by"></i>
              </span>
              </a>
            
          
        </div>
      </div>
    </div>
    <div class="license-icon iconfont"></div>
  </div>



              
                <div class="post-prevnext my-3">
                  <article class="post-prev col-6">
                    
                    
                      <a href="/2022/01/18/Hadoop%20_%20HDFS%20%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0%EF%BC%88%E4%BA%8C%EF%BC%89HDFS%20Java%20API%20%E7%8E%AF%E5%A2%83%E6%90%AD%E5%BB%BA%20_%20Java%E6%93%8D%E4%BD%9CHDFS%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9F%20_%20%E5%A4%9A%E6%A1%88%E4%BE%8B/" title="Hadoop _ HDFS 学习笔记（二）HDFS Java API 环境搭建 _ Java操作HDFS文件系统 _ 多案例">
                        <i class="iconfont icon-arrowleft"></i>
                        <span class="hidden-mobile">Hadoop _ HDFS 学习笔记（二）HDFS Java API 环境搭建 _ Java操作HDFS文件系统 _ 多案例</span>
                        <span class="visible-mobile">上一篇</span>
                      </a>
                    
                  </article>
                  <article class="post-next col-6">
                    
                    
                      <a href="/2022/01/17/Hadoop%20_%20MapReduce%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0%20_%20JavaAPI%E6%9B%B4%E6%8D%A2%E5%88%87%E7%89%87%E6%9C%BA%E5%88%B6%20_%20CombineTextInputFormat%20%E5%88%87%E7%89%87%20_%20%E8%AF%8D%E9%A2%91%E7%BB%9F%E8%AE%A1%E6%A1%88%E4%BE%8B/" title="Hadoop _ HDFS学习笔记（一）HDFS优缺点、NN+DN+2NN组成架构、block文件块 _ Shell操作 _ HDFS常用命令汇总">
                        <span class="hidden-mobile">Hadoop _ HDFS学习笔记（一）HDFS优缺点、NN+DN+2NN组成架构、block文件块 _ Shell操作 _ HDFS常用命令汇总</span>
                        <span class="visible-mobile">下一篇</span>
                        <i class="iconfont icon-arrowright"></i>
                      </a>
                    
                  </article>
                </div>
              
            </div>

            
          </article>
        </div>
      </div>
    </div>

    <div class="side-col d-none d-lg-block col-lg-2">
      
  <aside class="sidebar" style="margin-left: -1rem">
    <div id="toc">
  <p class="toc-header"><i class="iconfont icon-list"></i>&nbsp;目录</p>
  <div class="toc-body" id="toc-body"></div>
</div>



  </aside>


    </div>
  </div>
</div>





  



  



  



  



  







    

    
      <a id="scroll-top-button" aria-label="TOP" href="#" role="button">
        <i class="iconfont icon-arrowup" aria-hidden="true"></i>
      </a>
    

    
      <div class="modal fade" id="modalSearch" tabindex="-1" role="dialog" aria-labelledby="ModalLabel"
     aria-hidden="true">
  <div class="modal-dialog modal-dialog-scrollable modal-lg" role="document">
    <div class="modal-content">
      <div class="modal-header text-center">
        <h4 class="modal-title w-100 font-weight-bold">搜索</h4>
        <button type="button" id="local-search-close" class="close" data-dismiss="modal" aria-label="Close">
          <span aria-hidden="true">&times;</span>
        </button>
      </div>
      <div class="modal-body mx-3">
        <div class="md-form mb-5">
          <input type="text" id="local-search-input" class="form-control validate">
          <label data-error="x" data-success="v" for="local-search-input">关键词</label>
        </div>
        <div class="list-group" id="local-search-result"></div>
      </div>
    </div>
  </div>
</div>

    

    
  </main>

  <footer>
    <div class="footer-inner">
  
    <div class="footer-content">
       <a href="https://hexo.io" target="_blank" rel="nofollow noopener"><span>Hexo</span></a> <i class="iconfont icon-love"></i> <a href="https://github.com/fluid-dev/hexo-theme-fluid" target="_blank" rel="nofollow noopener"><span>Fluid</span></a> 
    </div>
  
  
  
  
</div>

  </footer>

  <!-- Scripts -->
  
  <script  src="https://lib.baomitu.com/nprogress/0.2.0/nprogress.min.js" ></script>
  <link  rel="stylesheet" href="https://lib.baomitu.com/nprogress/0.2.0/nprogress.min.css" />

  <script>
    NProgress.configure({"showSpinner":false,"trickleSpeed":100})
    NProgress.start()
    window.addEventListener('load', function() {
      NProgress.done();
    })
  </script>


<script  src="https://lib.baomitu.com/jquery/3.6.0/jquery.min.js" ></script>
<script  src="https://lib.baomitu.com/twitter-bootstrap/4.6.1/js/bootstrap.min.js" ></script>
<script  src="/js/events.js" ></script>
<script  src="/js/plugins.js" ></script>


  <script  src="https://lib.baomitu.com/typed.js/2.0.12/typed.min.js" ></script>
  <script>
    (function (window, document) {
      var typing = Fluid.plugins.typing;
      var subtitle = document.getElementById('subtitle');
      if (!subtitle || !typing) {
        return;
      }
      var text = subtitle.getAttribute('data-typed-text');
      
        typing(text);
      
    })(window, document);
  </script>




  
    <script  src="/js/img-lazyload.js" ></script>
  




  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/tocbot/4.18.2/tocbot.min.js', function() {
    var toc = jQuery('#toc');
    if (toc.length === 0 || !window.tocbot) { return; }
    var boardCtn = jQuery('#board-ctn');
    var boardTop = boardCtn.offset().top;

    window.tocbot.init({
      tocSelector     : '#toc-body',
      contentSelector : '.markdown-body',
      headingSelector : CONFIG.toc.headingSelector || 'h1,h2,h3,h4,h5,h6',
      linkClass       : 'tocbot-link',
      activeLinkClass : 'tocbot-active-link',
      listClass       : 'tocbot-list',
      isCollapsedClass: 'tocbot-is-collapsed',
      collapsibleClass: 'tocbot-is-collapsible',
      collapseDepth   : CONFIG.toc.collapseDepth || 0,
      scrollSmooth    : true,
      headingsOffset  : -boardTop
    });
    if (toc.find('.toc-list-item').length > 0) {
      toc.css('visibility', 'visible');
    }
  });
</script>


  <script src=https://lib.baomitu.com/clipboard.js/2.0.10/clipboard.min.js></script>

  <script>Fluid.plugins.codeWidget();</script>


  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/anchor-js/4.3.1/anchor.min.js', function() {
    window.anchors.options = {
      placement: CONFIG.anchorjs.placement,
      visible  : CONFIG.anchorjs.visible
    };
    if (CONFIG.anchorjs.icon) {
      window.anchors.options.icon = CONFIG.anchorjs.icon;
    }
    var el = (CONFIG.anchorjs.element || 'h1,h2,h3,h4,h5,h6').split(',');
    var res = [];
    for (var item of el) {
      res.push('.markdown-body > ' + item.trim());
    }
    if (CONFIG.anchorjs.placement === 'left') {
      window.anchors.options.class = 'anchorjs-link-left';
    }
    window.anchors.add(res.join(', '));
  });
</script>


  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/fancybox/3.5.7/jquery.fancybox.min.js', function() {
    Fluid.plugins.fancyBox();
  });
</script>


  <script>Fluid.plugins.imageCaption();</script>

  <script  src="/js/local-search.js" ></script>





<!-- 主题的启动项，将它保持在最底部 -->
<!-- the boot of the theme, keep it at the bottom -->
<script  src="/js/boot.js" ></script>


  

  <noscript>
    <div class="noscript-warning">博客在允许 JavaScript 运行的环境下浏览效果更佳</div>
  </noscript>
</body>
</html>
